{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.types import *\n",
    "from pyspark.sql import Row\n",
    "from pyspark.sql.functions import *\n",
    "spark = SparkSession.builder.appName(\n",
    "    \"HelloSpark\").master(\"local\").getOrCreate()\n",
    "spark.conf.set('spark.sql.repl.eagerEval.enabled', True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. map\n",
    "rdd = spark.sparkContext.parallelize([1, 2, 3, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2, 4, 6, 8]\n"
     ]
    }
   ],
   "source": [
    "result = rdd.map(lambda x:x*2).collect()\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "rdd = spark.sparkContext.parallelize([(\"jayChou\", 41), (\"burukeyou\", 23)])\n",
    "result = rdd.map(lambda row : Row(name=row[0], age=row[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(name='jayChou', age=41), Row(name='burukeyou', age=23)]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[name: string, age: bigint]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.createDataFrame(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# flatMap\n",
    "rdd = spark.sparkContext.parallelize([[1, 3, 4], [2, 3], [4, 5], [7, 8]])\n",
    "# 在 sparkSQL 中可以使用 explode 实现类似功能"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = spark.read.text(\"test.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+\n",
      "|               value|\n",
      "+--------------------+\n",
      "|There was no poss...|\n",
      "|                    |\n",
      "|I was glad of it:...|\n",
      "|                    |\n",
      "|The said Eliza, J...|\n",
      "|                    |\n",
      "|\"What does Bessie...|\n",
      "|                    |\n",
      "|\"Jane, I don't li...|\n",
      "|                    |\n",
      "|A breakfast-room ...|\n",
      "+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-RECORD 0-------------------------------------\n",
      " split(value, \\s+, -1) | [There, was, no, ... \n",
      "-RECORD 1-------------------------------------\n",
      " split(value, \\s+, -1) | []                   \n",
      "-RECORD 2-------------------------------------\n",
      " split(value, \\s+, -1) | [I, was, glad, of... \n",
      "-RECORD 3-------------------------------------\n",
      " split(value, \\s+, -1) | []                   \n",
      "-RECORD 4-------------------------------------\n",
      " split(value, \\s+, -1) | [The, said, Eliza... \n",
      "-RECORD 5-------------------------------------\n",
      " split(value, \\s+, -1) | []                   \n",
      "-RECORD 6-------------------------------------\n",
      " split(value, \\s+, -1) | [\"What, does, Bes... \n",
      "-RECORD 7-------------------------------------\n",
      " split(value, \\s+, -1) | []                   \n",
      "-RECORD 8-------------------------------------\n",
      " split(value, \\s+, -1) | [\"Jane,, I, don't... \n",
      "-RECORD 9-------------------------------------\n",
      " split(value, \\s+, -1) | []                   \n",
      "-RECORD 10------------------------------------\n",
      " split(value, \\s+, -1) | [A, breakfast-roo... \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 切分\n",
    "split_df = df.select(split('value', '\\s+'))\n",
    "split_df.show(vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(split(value, \\s+, -1)=['There', 'was', 'no', 'possibility', 'of', 'taking', 'a', 'walk', 'that', 'day.', 'We', 'had', 'been', 'wandering,', 'indeed,', 'in', 'the', 'leafless', 'shrubbery', 'an', 'hour', 'in', 'the', 'morning;', 'but', 'since', 'dinner', '(Mrs.', 'Reed,', 'when', 'there', 'was', 'no', 'company,', 'dined', 'early)', 'the', 'cold', 'winter', 'wind', 'had', 'brought', 'with', 'it', 'clouds', 'so', 'sombre,', 'and', 'a', 'rain', 'so', 'penetrating,', 'that', 'further', 'out-door', 'exercise', 'was', 'now', 'out', 'of', 'the', 'question.']),\n",
       " Row(split(value, \\s+, -1)=['']),\n",
       " Row(split(value, \\s+, -1)=['I', 'was', 'glad', 'of', 'it:', 'I', 'never', 'liked', 'long', 'walks,', 'especially', 'on', 'chilly', 'afternoons:', 'dreadful', 'to', 'me', 'was', 'the', 'coming', 'home', 'in', 'the', 'raw', 'twilight,', 'with', 'nipped', 'fingers', 'and', 'toes,', 'and', 'a', 'heart', 'saddened', 'by', 'the', 'chidings', 'of', 'Bessie,', 'the', 'nurse,', 'and', 'humbled', 'by', 'the', 'consciousness', 'of', 'my', 'physical', 'inferiority', 'to', 'Eliza,', 'John,', 'and', 'Georgiana', 'Reed.']),\n",
       " Row(split(value, \\s+, -1)=['']),\n",
       " Row(split(value, \\s+, -1)=['The', 'said', 'Eliza,', 'John,', 'and', 'Georgiana', 'were', 'now', 'clustered', 'round', 'their', 'mama', 'in', 'the', 'drawing-room:', 'she', 'lay', 'reclined', 'on', 'a', 'sofa', 'by', 'the', 'fireside,', 'and', 'with', 'her', 'darlings', 'about', 'her', '(for', 'the', 'time', 'neither', 'quarrelling', 'nor', 'crying)', 'looked', 'perfectly', 'happy.', 'Me,', 'she', 'had', 'dispensed', 'from', 'joining', 'the', 'group;', 'saying,', '\"She', 'regretted', 'to', 'be', 'under', 'the', 'necessity', 'of', 'keeping', 'me', 'at', 'a', 'distance;', 'but', 'that', 'until', 'she', 'heard', 'from', 'Bessie,', 'and', 'could', 'discover', 'by', 'her', 'own', 'observation,', 'that', 'I', 'was', 'endeavouring', 'in', 'good', 'earnest', 'to', 'acquire', 'a', 'more', 'sociable', 'and', 'childlike', 'disposition,', 'a', 'more', 'attractive', 'and', 'sprightly', 'manner--', 'something', 'lighter,', 'franker,', 'more', 'natural,', 'as', 'it', 'were--she', 'really', 'must', 'exclude', 'me', 'from', 'privileges', 'intended', 'only', 'for', 'contented,', 'happy,', 'little', 'children.\"'])]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "split_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "split_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>word</th></tr>\n",
       "<tr><td>There</td></tr>\n",
       "<tr><td>was</td></tr>\n",
       "<tr><td>no</td></tr>\n",
       "<tr><td>possibility</td></tr>\n",
       "<tr><td>of</td></tr>\n",
       "<tr><td>taking</td></tr>\n",
       "<tr><td>a</td></tr>\n",
       "<tr><td>walk</td></tr>\n",
       "<tr><td>that</td></tr>\n",
       "<tr><td>day.</td></tr>\n",
       "<tr><td>We</td></tr>\n",
       "<tr><td>had</td></tr>\n",
       "<tr><td>been</td></tr>\n",
       "<tr><td>wandering,</td></tr>\n",
       "<tr><td>indeed,</td></tr>\n",
       "<tr><td>in</td></tr>\n",
       "<tr><td>the</td></tr>\n",
       "<tr><td>leafless</td></tr>\n",
       "<tr><td>shrubbery</td></tr>\n",
       "<tr><td>an</td></tr>\n",
       "</table>\n",
       "only showing top 20 rows\n"
      ],
      "text/plain": [
       "+-----------+\n",
       "|       word|\n",
       "+-----------+\n",
       "|      There|\n",
       "|        was|\n",
       "|         no|\n",
       "|possibility|\n",
       "|         of|\n",
       "|     taking|\n",
       "|          a|\n",
       "|       walk|\n",
       "|       that|\n",
       "|       day.|\n",
       "|         We|\n",
       "|        had|\n",
       "|       been|\n",
       "| wandering,|\n",
       "|    indeed,|\n",
       "|         in|\n",
       "|        the|\n",
       "|   leafless|\n",
       "|  shrubbery|\n",
       "|         an|\n",
       "+-----------+\n",
       "only showing top 20 rows"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 切分\n",
    "explode_df = df.select(explode(split('value', '\\s+')).alias('word'))\n",
    "explode_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "344"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "explode_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>word</th><th>count</th></tr>\n",
       "<tr><td>taking</td><td>3</td></tr>\n",
       "<tr><td>humbled</td><td>1</td></tr>\n",
       "<tr><td>indeed,</td><td>1</td></tr>\n",
       "<tr><td>Georgiana</td><td>2</td></tr>\n",
       "<tr><td>question.</td><td>1</td></tr>\n",
       "<tr><td>observation,</td><td>1</td></tr>\n",
       "<tr><td>nurse,</td><td>1</td></tr>\n",
       "<tr><td>could</td><td>1</td></tr>\n",
       "<tr><td>Turk;</td><td>1</td></tr>\n",
       "<tr><td>raw</td><td>1</td></tr>\n",
       "<tr><td>perfectly</td><td>1</td></tr>\n",
       "<tr><td>Reed.</td><td>1</td></tr>\n",
       "<tr><td>it:</td><td>1</td></tr>\n",
       "<tr><td>drawing-room:</td><td>1</td></tr>\n",
       "<tr><td>mama</td><td>1</td></tr>\n",
       "<tr><td>by</td><td>4</td></tr>\n",
       "<tr><td>must</td><td>1</td></tr>\n",
       "<tr><td>inferiority</td><td>1</td></tr>\n",
       "<tr><td>breakfast-room</td><td>1</td></tr>\n",
       "<tr><td>you</td><td>1</td></tr>\n",
       "</table>\n",
       "only showing top 20 rows\n"
      ],
      "text/plain": [
       "+--------------+-----+\n",
       "|          word|count|\n",
       "+--------------+-----+\n",
       "|        taking|    3|\n",
       "|       humbled|    1|\n",
       "|       indeed,|    1|\n",
       "|     Georgiana|    2|\n",
       "|     question.|    1|\n",
       "|  observation,|    1|\n",
       "|        nurse,|    1|\n",
       "|         could|    1|\n",
       "|         Turk;|    1|\n",
       "|           raw|    1|\n",
       "|     perfectly|    1|\n",
       "|         Reed.|    1|\n",
       "|           it:|    1|\n",
       "| drawing-room:|    1|\n",
       "|          mama|    1|\n",
       "|            by|    4|\n",
       "|          must|    1|\n",
       "|   inferiority|    1|\n",
       "|breakfast-room|    1|\n",
       "|           you|    1|\n",
       "+--------------+-----+\n",
       "only showing top 20 rows"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_count_result = explode_df.groupBy('word').count()\n",
    "word_count_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>name</th><th>age</th></tr>\n",
       "<tr><td>jack</td><td>18</td></tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "+----+---+\n",
       "|name|age|\n",
       "+----+---+\n",
       "|jack| 18|\n",
       "+----+---+"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 过滤\n",
    "rdd = spark.sparkContext.parallelize([('tom', 20), ('jack', 18)])\n",
    "df = rdd.toDF(['name', 'age'])\n",
    "df.where('age < 20')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>name</th><th>age</th></tr>\n",
       "<tr><td>jack</td><td>18</td></tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "+----+---+\n",
       "|name|age|\n",
       "+----+---+\n",
       "|jack| 18|\n",
       "+----+---+"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 或者\n",
    "df.filter('age < 20')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>name</th><th>age</th></tr>\n",
       "<tr><td>jack</td><td>18</td></tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "+----+---+\n",
       "|name|age|\n",
       "+----+---+\n",
       "|jack| 18|\n",
       "+----+---+"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.filter(col('age') < 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 切分randomSplit\n",
    "rdd = spark.sparkContext.parallelize([(i, 'fake') for i in range(100)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>num</th><th>date</th></tr>\n",
       "<tr><td>0</td><td>fake</td></tr>\n",
       "<tr><td>1</td><td>fake</td></tr>\n",
       "<tr><td>2</td><td>fake</td></tr>\n",
       "<tr><td>3</td><td>fake</td></tr>\n",
       "<tr><td>4</td><td>fake</td></tr>\n",
       "<tr><td>5</td><td>fake</td></tr>\n",
       "<tr><td>6</td><td>fake</td></tr>\n",
       "<tr><td>7</td><td>fake</td></tr>\n",
       "<tr><td>8</td><td>fake</td></tr>\n",
       "<tr><td>9</td><td>fake</td></tr>\n",
       "<tr><td>10</td><td>fake</td></tr>\n",
       "<tr><td>11</td><td>fake</td></tr>\n",
       "<tr><td>12</td><td>fake</td></tr>\n",
       "<tr><td>13</td><td>fake</td></tr>\n",
       "<tr><td>14</td><td>fake</td></tr>\n",
       "<tr><td>15</td><td>fake</td></tr>\n",
       "<tr><td>16</td><td>fake</td></tr>\n",
       "<tr><td>17</td><td>fake</td></tr>\n",
       "<tr><td>18</td><td>fake</td></tr>\n",
       "<tr><td>19</td><td>fake</td></tr>\n",
       "</table>\n",
       "only showing top 20 rows\n"
      ],
      "text/plain": [
       "+---+----+\n",
       "|num|date|\n",
       "+---+----+\n",
       "|  0|fake|\n",
       "|  1|fake|\n",
       "|  2|fake|\n",
       "|  3|fake|\n",
       "|  4|fake|\n",
       "|  5|fake|\n",
       "|  6|fake|\n",
       "|  7|fake|\n",
       "|  8|fake|\n",
       "|  9|fake|\n",
       "| 10|fake|\n",
       "| 11|fake|\n",
       "| 12|fake|\n",
       "| 13|fake|\n",
       "| 14|fake|\n",
       "| 15|fake|\n",
       "| 16|fake|\n",
       "| 17|fake|\n",
       "| 18|fake|\n",
       "| 19|fake|\n",
       "+---+----+\n",
       "only showing top 20 rows"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = rdd.toDF(['num', 'date'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[+---+----+\n",
       " |num|date|\n",
       " +---+----+\n",
       " |  0|fake|\n",
       " |  1|fake|\n",
       " |  3|fake|\n",
       " |  4|fake|\n",
       " |  5|fake|\n",
       " |  7|fake|\n",
       " | 10|fake|\n",
       " | 11|fake|\n",
       " | 12|fake|\n",
       " | 16|fake|\n",
       " | 17|fake|\n",
       " | 18|fake|\n",
       " | 20|fake|\n",
       " | 22|fake|\n",
       " | 25|fake|\n",
       " | 26|fake|\n",
       " | 27|fake|\n",
       " | 31|fake|\n",
       " | 33|fake|\n",
       " | 36|fake|\n",
       " +---+----+\n",
       " only showing top 20 rows,\n",
       " +---+----+\n",
       " |num|date|\n",
       " +---+----+\n",
       " |  2|fake|\n",
       " |  6|fake|\n",
       " |  8|fake|\n",
       " |  9|fake|\n",
       " | 13|fake|\n",
       " | 14|fake|\n",
       " | 15|fake|\n",
       " | 19|fake|\n",
       " | 21|fake|\n",
       " | 23|fake|\n",
       " | 24|fake|\n",
       " | 28|fake|\n",
       " | 29|fake|\n",
       " | 30|fake|\n",
       " | 32|fake|\n",
       " | 34|fake|\n",
       " | 35|fake|\n",
       " | 39|fake|\n",
       " | 42|fake|\n",
       " | 43|fake|\n",
       " +---+----+\n",
       " only showing top 20 rows]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfs = df.randomSplit([0.7, 0.3], seed=42)\n",
    "dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(65, 35)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfs[0].count(), dfs[1].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39\n",
      "+---+----+\n",
      "|num|data|\n",
      "+---+----+\n",
      "|6  |fake|\n",
      "|7  |fake|\n",
      "|8  |fake|\n",
      "|9  |fake|\n",
      "|10 |fake|\n",
      "|14 |fake|\n",
      "|15 |fake|\n",
      "|19 |fake|\n",
      "|20 |fake|\n",
      "|23 |fake|\n",
      "|29 |fake|\n",
      "|34 |fake|\n",
      "|35 |fake|\n",
      "|38 |fake|\n",
      "|40 |fake|\n",
      "|41 |fake|\n",
      "|42 |fake|\n",
      "|51 |fake|\n",
      "|52 |fake|\n",
      "|59 |fake|\n",
      "|61 |fake|\n",
      "|63 |fake|\n",
      "|64 |fake|\n",
      "|69 |fake|\n",
      "|71 |fake|\n",
      "|73 |fake|\n",
      "|75 |fake|\n",
      "|77 |fake|\n",
      "|78 |fake|\n",
      "|81 |fake|\n",
      "|82 |fake|\n",
      "|83 |fake|\n",
      "|90 |fake|\n",
      "|92 |fake|\n",
      "|93 |fake|\n",
      "|95 |fake|\n",
      "|96 |fake|\n",
      "|97 |fake|\n",
      "|99 |fake|\n",
      "+---+----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 取样\n",
    "rdd = spark.sparkContext.parallelize([(i, 'fake') for i in range(1, 100)])\n",
    " \n",
    "df = rdd.toDF(['num', 'data'])\n",
    " \n",
    "sample_data = df.sample(0.4)\n",
    "print(sample_data.count())\n",
    "sample_data.show(100, truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>name</th><th>age</th></tr>\n",
       "<tr><td>burukeyou</td><td>23</td></tr>\n",
       "<tr><td>jayChou</td><td>41</td></tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "+---------+---+\n",
       "|     name|age|\n",
       "+---------+---+\n",
       "|burukeyou| 23|\n",
       "|  jayChou| 41|\n",
       "+---------+---+"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 排序\n",
    "rdd = spark.sparkContext.parallelize([(\"jayChou\", 41), (\"burukeyou\", 23)])\n",
    " \n",
    "df = spark.createDataFrame(rdd.map(lambda row: Row(name=row[0], age=row[1])))\n",
    "df.orderBy('age')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>name</th><th>age</th></tr>\n",
       "<tr><td>jayChou</td><td>41</td></tr>\n",
       "<tr><td>burukeyou</td><td>23</td></tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "+---------+---+\n",
       "|     name|age|\n",
       "+---------+---+\n",
       "|  jayChou| 41|\n",
       "|burukeyou| 23|\n",
       "+---------+---+"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 去重\n",
    "rdd = spark.sparkContext.parallelize([(\"jayChou\", 41), (\"burukeyou\", 23), (\"burukeyou\", 23)])\n",
    "df = spark.createDataFrame(rdd.map(lambda row: Row(name=row[0], age=row[1])))\n",
    "df.drop_duplicates(['name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>name</th></tr>\n",
       "<tr><td>jayChou</td></tr>\n",
       "<tr><td>burukeyou</td></tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "+---------+\n",
       "|     name|\n",
       "+---------+\n",
       "|  jayChou|\n",
       "|burukeyou|\n",
       "+---------+"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.select('name').distinct()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.select('name').take(1)\n",
    "# 可以去看看 take 的实现。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.4 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "d4d1e4263499bec80672ea0156c357c1ee493ec2b1c70f0acce89fc37c4a6abe"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
